/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */
/*
 * Copyright (c) 2009 INRIA
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation;
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *
 * Author: Guillaume Seguin <guillaume@segu.in>
 */

#ifndef MULTITHREADED_SIMULATOR_IMPL_H
#define MULTITHREADED_SIMULATOR_IMPL_H

#include "ns3/simulator-impl.h"
#include "ns3/scheduler.h"
#include "ns3/event-impl.h"
#include "multithreading-partition.h"

#include "ns3/ptr.h"
#include "ns3/assert.h"
#include "ns3/log.h"
//#include "ns3/barrier.h"
#include "lbts-message.h"

#ifdef NS3_MPI
#include <mpi.h>
#endif

#include <pthread.h>
#include <semaphore.h>
#include <list>
#include <vector>
#include <queue>
#include "barrier.h"

#define CLOCK_DEBUGGING 0

#define LOCKLESS_SHARED_PARTITIONS 1

namespace ns3 {

inline uint64_t NextPartitionEventTs (MultiThreadingPartition *partition);

class MultiThreadedSimulatorImpl : public SimulatorImpl
{
public:
	enum SynchronizationMode {
	  SYNC_BEST_EFFORT, /** Make a best effort to keep synced to real-time */
	  SYNC_HARD_LIMIT, /** Keep to real-time within a tolerance or die trying */
	};

  static TypeId GetTypeId (void);

  MultiThreadedSimulatorImpl ();
  ~MultiThreadedSimulatorImpl ();
  
  enum BarrierType {
    BARRIER_POSIX,
    BARRIER_GLOBALSPIN,
    BARRIER_TREESPIN
  };

  void DoDispose (void);
  void Destroy (void);
  bool IsFinished (void) const;
  Time Next (void) const;
  void Stop (void);
  void Stop (Time const &time);
  EventId Schedule (Time const &time, EventImpl *event);
  void ScheduleWithContext (uint32_t context, Time const &time, EventImpl *event);
//  void ScheduleWithSubContext(uint32_t context, uint32_t subContext, Time const &time, EventImpl *event);
  EventId ScheduleNow (EventImpl *event);
  EventId ScheduleDestroy (EventImpl *event);
  void Remove (const EventId &ev);
  void Cancel (const EventId &ev);
  bool IsExpired (const EventId &ev) const;
  void Run (void);
  void RunOneEvent (void);
  Time Now (void) const;
  Time RealtimeNow (void) const;
  Time GetDelayLeft (const EventId &id) const;
  Time GetMaximumSimulationTime (void) const;
  uint32_t GetContext (void) const;
  uint32_t GetSystemId (void) const;
  void SetScheduler (ObjectFactory schedulerFactory);
  void SetPartitionScheduler (MultiThreadingPartition *partition, ObjectFactory schedulerFactory);
  /**
   * Registers a partition to simulator, associated with the given context
   *
   * \param context Context of the given partition
   * \param partition The partition to add
   */
  void AddPartition (uint32_t context, MultiThreadingPartition *partition);
  uint32_t AddSubPartition (uint32_t nodeId, uint32_t subContext);
  MultiThreadingPartition* GetMappedPartition(uint32_t context, uint32_t subContext);

private:
  inline void ProcessPartitionEvent (MultiThreadingPartition *partition);
  inline uint32_t RunPartitionUntil (MultiThreadingPartition *partition, uint64_t maxTs);
  uint32_t ProcessPartition (MultiThreadingPartition *partition);

  void PushEventMessage (MultiThreadingPartition *partition, uint32_t from, uint64_t timestamp, EventImpl *event);

  #if LOCKLESS_SHARED_PARTITIONS
  struct LockLessPartition {
    int threadId;
    MultiThreadingPartition *partition;
  };

  struct SharedPartitionsSet {
    int currentPartition;
    std::vector<struct LockLessPartition> partitions;
  };
  #else /* LOCKLESS_SHARED_PARTITIONS */
  struct SharedPartitionsSet {
    uint32_t currentPartition;
    pthread_mutex_t lock;
    std::vector<MultiThreadingPartition *> partitions;
  };
  #endif /* LOCKLESS_SHARED_PARTITIONS */

  struct ThreadContext {
    uint32_t id;
    pthread_t thread;
    MultiThreadedSimulatorImpl *simulator;
    std::list<MultiThreadingPartition *> dedicatedPartitions;
    struct SharedPartitionsSet *sharedPartitions;
  };

  struct ThreadContext *m_threads;
  struct InitialEvent {
    uint32_t context;
    uint64_t timestamp;
    EventImpl *event;
  };
  typedef std::list<EventId> DestroyEvents;
  typedef std::queue<struct InitialEvent> InitialEvents;
  typedef std::map<int, MultiThreadingPartition *> MultiThreadingPartitions;

  static void* RunThread (void *data);
  uint64_t GetMaximumSimulationTs (void) const;
  void DoRunThread (struct ThreadContext *context);
  void StopOnePartition (void);

  uint32_t m_uid;
  uint32_t m_globalUid;
  uint64_t m_globalTs;
  mutable pthread_mutex_t m_globalEventsLock;
  Scheduler *m_globalEvents;

  mutable pthread_mutex_t m_destroyEventsLock;
  DestroyEvents m_destroyEvents;

  InitialEvents m_initialEvents;
  MultiThreadingPartitions m_partitions;
  uint32_t m_threadsCount;
  ObjectFactory m_schedulerFactory;
  bool m_running;
  int m_nPartitions;
  int m_nStoppedPartitions;
  int m_nEvents;
  
  bool m_isRealTime;
  bool m_isDistributed;

  uint32_t m_threadDedicatedPercent;
  MultiThreadingPartition *GetNextSharedPartition (struct ThreadContext *context);

  struct SharedPartitionsSet *m_sharedPartitionsSets;
  uint32_t m_nSharedPartitionsSets;
  uint32_t m_nSharedPartitions;

  BarrierType m_barrierType;
  Barrier *m_barrier;
  uint64_t m_maxTs;

  uint32_t m_rank;
  uint32_t m_numRanks;
  LbtsMessage* m_lbtsMessages;

  SynchronizationMode m_synchronizationMode;
  Time m_hardLimit;

  typedef std::map<uint32_t, std::map<uint32_t, uint32_t> > ContextToPartitionMap;
  ContextToPartitionMap m_contextToPartitionMap;
  mutable pthread_mutex_t m_addPartitionLock;
};

} // namespace ns3

#endif /* MULTITHREADED_SIMULATOR_IMPL_H */
